//////////////////////////////////////////////////////////////////////
//
//	Crytek CryENGINE Source code
//
//	File:FileReadSequencer.cpp
//  Description: Loading file data ahead for fast load from DVD
//    Usually all CVars has to be set into release state for proper work of file sequencer:
//
//	History:
//	  -May 24,2010: Created by Vladimir Kajalin
//
//////////////////////////////////////////////////////////////////////

#include "StdAfx.h"
#include "CryPak.h"
#include <ILog.h>
#include <ISystem.h>
#include <IPlatformOS.h>
#include <IConsole.h>
#include <ITimer.h>
#include <IPerfHud.h>
#include "zlib/zlib.h"				// crc32()
#include "md5.h"

#if defined(PS3) || defined(LINUX)
#include "System.h"
#include <unistd.h>
#include <sys/stat.h>					// fstat, fileno
#endif





#include "ReadSequencerThread.h"

int CFileReadSequencer::m_nSequencerMode=0;
int CFileReadSequencer::m_nSequencerProfile=0;
string CFileReadSequencer::m_sSectionName;
DWORD CFileReadSequencer::m_dwTargetThread=0;
CryCriticalSection CFileReadSequencer::m_csFread;
FILE * CFileReadSequencer::m_hWriteFile=0;
int CFileReadSequencer::m_nCurPageStartOffset = 0;
int CFileReadSequencer::m_nCurPageBytes = 0;
class CReadSequencerThread * CFileReadSequencer::m_pReadThread = 0;
std::map<FILE*,CFileReadSequencer::SFileAccessStats> CFileReadSequencer::m_mapOpenFiles;
CryCriticalSection CFileReadSequencer::m_csMapOpenFiles;
CTimeValue CFileReadSequencer::m_sectionStartTime;
int CFileReadSequencer::m_nMessageOrder = 0; 
CFileReadSequencer::SFileAccessStats CFileReadSequencer::m_seqStats;

struct SDeferredBlock
{
  SSeqHeader header;
  PodArray<byte> data;
};

PodArray<struct SDeferredBlock*> CFileReadSequencer::m_arrDeferredBlocks;

void CFileReadSequencer::OnFopen(FILE * f, const char * file, const char * mode)
{
  if(m_nSequencerProfile==0)
    return;

  AUTO_LOCK(m_csMapOpenFiles);

  if(m_mapOpenFiles[f].sName.length() && strcmp(m_mapOpenFiles[f].sName.c_str(),file)!=0)
    CFileReadSequencer::Log("CFileReadSequencer::OnFopen: %s >> %s", m_mapOpenFiles[f].sName.c_str(), file);

  m_mapOpenFiles[f].sName = file;
}

bool CFileReadSequencer::IsProfiling()
{
  return CFileReadSequencer::m_nSequencerProfile != 0;
}

int CFileReadSequencer::Fread(void *pData, size_t nSize, size_t nCount, FILE *hFile)
{
  float fStartTime = gEnv->pTimer->GetAsyncCurTime();

  int nItemsRead=0;

  if( !m_pReadThread && !m_hWriteFile )
  {
    m_seqStats.off.nBytes += nCount*nSize;
    m_seqStats.off.nReads ++;

    nItemsRead = CReadSequencerThread::FreadLowLevel(pData, nSize, nCount, hFile);

    float fTimeDelta = max(0.f, gEnv->pTimer->GetAsyncCurTime() - fStartTime);

    if(SFileAccessStats * pInfo = GetFileInfo(hFile))
    {
      pInfo->off.fTime += fTimeDelta;
      pInfo->off.nBytes += nSize*nCount;
      pInfo->off.nReads ++;
    }

    CFileReadSequencer::m_seqStats.off.fTime += fTimeDelta;

    return nItemsRead;
  }

  AUTO_LOCK(m_csFread);

  if(m_dwTargetThread == ~0)
    m_dwTargetThread = GetCurrentThreadId();

  if(GetCurrentThreadId() != m_dwTargetThread)
  {
    m_seqStats.oth.nBytes += nCount*nSize;
    m_seqStats.oth.nReads ++;

    nItemsRead = CReadSequencerThread::FreadLowLevel(pData, nSize, nCount, hFile);

    float fTimeDelta = max(0.f, gEnv->pTimer->GetAsyncCurTime() - fStartTime);

    if(SFileAccessStats * pInfo = GetFileInfo(hFile))
    {
      pInfo->oth.fTime += fTimeDelta;
      pInfo->oth.nBytes += nSize*nCount;
      pInfo->oth.nReads ++;
    }

    CFileReadSequencer::m_seqStats.oth.fTime += fTimeDelta;

    return nItemsRead;
  }

  if(nSize*nCount > 8*1024*1024)
  {
    const char * pFileName = GetFileName(hFile);

    CFileReadSequencer::Log( "Block is bigger than %d MB: %.1f MB, %s", 
      8, float(nSize*nCount)/1024.f/1024.f, pFileName );
  }

  if(m_nSequencerMode==2) // create sequencer file
  {
    int nSeek = ::ftell(hFile);

    nItemsRead = CReadSequencerThread::FreadLowLevel(pData, nSize, nCount, hFile);

    if(m_hWriteFile)
    {
      char * pMarker = "CRY";
      ::fwrite(pMarker, 4, 1, m_hWriteFile);

      ::fwrite(&nSize, sizeof(nSize), 1, m_hWriteFile);
      ::fwrite(&nCount, sizeof(nCount), 1, m_hWriteFile);
      ::fwrite(&nItemsRead, sizeof(nItemsRead), 1, m_hWriteFile);

      ::fwrite(&nSeek, sizeof(nSeek), 1, m_hWriteFile);

      bool bBigBlock = (int)(nCount*nSize) > gEnv->pConsole->GetCVar("sys_FileReadSequencerInlineBlockMaxSizeKB")->GetIVal()*1024;
      static bool bLastBigBlock = false;

      int nMaxBytesInPage = (m_seqStats.ok.nBytes>1024*1024) ? 4*1024*1024 : 4*1024*1024/4;

      {
        int nZero = -1; // by default -1 means here - from now read until the end
        ::fwrite(&nZero, sizeof(nZero), 1, m_hWriteFile);
      }

      if((int)(m_nCurPageBytes + nCount*nSize + sizeof(SSeqHeader)) > nMaxBytesInPage || bBigBlock || bLastBigBlock)
      {
        SSeqHeader h;
        int nMemOffset = (int)((byte*)&h.nBlockGroupSize - (byte*)&h);
        ::fseek(m_hWriteFile, m_nCurPageStartOffset + nMemOffset, SEEK_SET);
        ::fwrite(&m_nCurPageBytes, sizeof(m_nCurPageBytes), 1, m_hWriteFile);
        ::fseek(m_hWriteFile, 0, SEEK_END);

        m_nCurPageStartOffset = ::ftell(m_hWriteFile) - nMemOffset - 4;

        const char * pFileName = GetFileName(hFile);
        CFileReadSequencer::Log( "Page restart: %s, page size=%d, read=%d - %s", 
          bBigBlock ? "BB_YES" : "BB_NO", m_nCurPageBytes, nCount*nSize, pFileName);

        m_nCurPageBytes = 0;
      }

#ifdef FS_DEBUG
      static char szCallStack[1024]="";
      strcpy_s(szCallStack, sizeof(szCallStack), gEnv->pSystem->GetLoadingProfilerCallstack());
      ::fwrite(szCallStack, sizeof(szCallStack), 1, m_hWriteFile);
      static char szFileName[1024]="";
      strcpy_s(szFileName, sizeof(szFileName), GetFileName(hFile));
      ::fwrite(szFileName, sizeof(szFileName), 1, m_hWriteFile);
#endif
     
      if(bBigBlock)
      { // store reference
        const char * pFileName = GetFileName(hFile);
        
        int nFileNameLen = strlen(pFileName);
        ::fwrite(&nFileNameLen, sizeof(nFileNameLen), 1, m_hWriteFile);

        ::fwrite(pFileName, nFileNameLen, 1, m_hWriteFile);
      }
      else
      { // store real data
        int nFileNameLen = 0;
        ::fwrite(&nFileNameLen, sizeof(nFileNameLen), 1, m_hWriteFile);

        ::fwrite(pData, nSize, nCount, m_hWriteFile);
      }

      ::fflush(m_hWriteFile);

      m_nCurPageBytes += nCount*nSize + sizeof(SSeqHeader);

			m_seqStats.ok.nBytes += nCount*nSize;
      m_seqStats.ok.nReads ++;

      bLastBigBlock = bBigBlock;
    }
  }

  if(m_nSequencerMode==1) // load from sequencer file
  {
    int nMissesNum = 0;

    SSeqHeader seqHeaderMiss;

    int nSeek = ::ftell(hFile);

    // search in deferred
    for(int i=0; i<m_arrDeferredBlocks.Count(); i++)
    {
      SDeferredBlock * & pItem = m_arrDeferredBlocks[i];

      if(pItem->header.nSize != nSize || pItem->header.nCount != nCount || pItem->header.nSeek != nSeek)
        continue;

      assert(nSize*nCount == pItem->data.Count());
      memcpy(pData, pItem->data.GetElements(), nSize*nCount);
      nItemsRead = nCount;

      SAFE_DELETE(pItem);
      m_arrDeferredBlocks.Delete(i);

      const char * pFileName = GetFileName(hFile);
      CFileReadSequencer::Log( "Block found in deferred: %d bytes, file: %s", nCount*nSize, pFileName );

      break;
    }

    if(!nItemsRead)
      while(m_pReadThread)
    {
      SSeqHeader seqHeader;

      if(m_pReadThread->Fread(&seqHeader, sizeof(seqHeader), 1, hFile) != 1)
      {
        LogStatus("STREAM_ENDED", hFile);
        SAFE_DELETE(m_pReadThread);
      }

      if(m_pReadThread)
        if(strncmp(seqHeader.sMarker, "CRY", sizeof(seqHeader.sMarker)))
      {
        LogStatus("STREAM_MARKER_ERROR", hFile);
        SAFE_DELETE(m_pReadThread);
      }

      bool bFound = (m_pReadThread!=0);
      if(m_pReadThread)
        if(seqHeader.nSize != nSize || seqHeader.nCount != nCount || seqHeader.nSeek != nSeek)
      {
        if(!seqHeaderMiss.nSize)
          seqHeaderMiss = seqHeader;

        bFound = false;

        if(!nMissesNum)
        {
          const char * pFileName = GetFileName(hFile);
          CFileReadSequencer::Log( "Miss detected: %d bytes, file: %s", 
            nCount*nSize, pFileName );
        }

        nMissesNum++;
      }

      if(m_pReadThread)
        nItemsRead = seqHeader.nItemsRead;

      if(m_pReadThread)
      {
        if(!bFound)
        {
          SDeferredBlock * pMiss = new SDeferredBlock;
          pMiss->header = seqHeader;
          pMiss->data.PreAllocate(seqHeader.nSize*seqHeader.nCount, seqHeader.nSize*seqHeader.nCount);

          if(m_pReadThread->Fread(pMiss->data.GetElements(), seqHeader.nSize, seqHeader.nCount, hFile) != seqHeader.nCount)
          {
            LogStatus("STREAM_READ_ERROR", hFile);
            SAFE_DELETE(m_pReadThread);
          }

          m_arrDeferredBlocks.Add(pMiss);

          if(nMissesNum>120)
          { // consider data as not available in sequencer file - read from real source
            m_seqStats.skip.nBytes += nCount*nSize;
            m_seqStats.skip.nReads ++;

            nItemsRead = CReadSequencerThread::FreadLowLevel(pData, nSize, nCount, hFile);

            float fTimeDelta = max(0.f, gEnv->pTimer->GetAsyncCurTime() - fStartTime);

            if(SFileAccessStats * pInfo = GetFileInfo(hFile))
            {
              pInfo->skip.fTime += fTimeDelta;
              pInfo->skip.nBytes += nSize*nCount;
              pInfo->skip.nReads ++;

              CFileReadSequencer::Log( "Miss read from original location: %d bytes, file: %s", nCount*nSize, pInfo->sName.c_str() );
            }

            CFileReadSequencer::m_seqStats.skip.fTime += fTimeDelta;

            return nItemsRead;
          }
        }
        else
        {
          if(m_pReadThread->Fread(bFound ? pData : NULL, seqHeader.nSize, seqHeader.nCount, hFile) != seqHeader.nCount)
          {
            LogStatus("STREAM_READ_ERROR", hFile);
            SAFE_DELETE(m_pReadThread);
          }
        }

        if(m_pReadThread)
        {
          if(bFound)
          {
            m_seqStats.ok.nBytes += nItemsRead*seqHeader.nSize;
            m_seqStats.ok.nReads++;
          }
          else
          {
            m_seqStats.skip.nBytes += nItemsRead*seqHeader.nSize;
            m_seqStats.skip.nReads++;

            if(SFileAccessStats * pInfo = GetFileInfo(hFile))
            {
              pInfo->skip.nBytes += nItemsRead*seqHeader.nSize;
              pInfo->skip.nReads ++;
            }
          }
        }
      }

      if(bFound)
      {
        if(nMissesNum)
        {
          seqHeaderMiss.Reset(); // miss fixed

          const char * pFileName = GetFileName(hFile);

          CFileReadSequencer::Log( "Miss fixed: %d blocks, %d bytes, file: %s", 
            nMissesNum, nCount*nSize, pFileName );
        
          nMissesNum = 0;
        }

        break;
      }
    }

    if(m_pReadThread && nItemsRead)
    {
      ::fseek(hFile, nSize*nCount, SEEK_CUR);
    }
    else if(seqHeaderMiss.nSize)
    {
      const char * pFileName = GetFileName(hFile);

      if(nMissesNum)
      {
        seqHeaderMiss.Reset(); // miss fixed

        CFileReadSequencer::Log( "DATA NOT FOUND, Miss not fixed: %d blocks, %d bytes, file: %s", 
          nMissesNum, nCount*nSize, pFileName );

        nMissesNum = 0;
      }
      else
        CFileReadSequencer::Log( "DATA NOT FOUND, No miss, File: %s", 
        pFileName );
    }
  }

  float fTimeDelta = max(0.f, gEnv->pTimer->GetAsyncCurTime() - fStartTime);
  
  if(SFileAccessStats * pInfo = GetFileInfo(hFile))
  {
    pInfo->ok.fTime += fTimeDelta;
    pInfo->ok.nBytes += nSize*nCount;
    pInfo->ok.nReads ++;
  }

  CFileReadSequencer::m_seqStats.ok.fTime += fTimeDelta;

  return nItemsRead;
}

void CFileReadSequencer::CloseSeqFile()
{
  // close file operations
  SAFE_DELETE(m_pReadThread);
  if(m_hWriteFile)
  {
    ::fclose(m_hWriteFile);
    m_hWriteFile = 0;
  }
}

const char * CFileReadSequencer::GetFileName(FILE * hFile)
{
  if(m_nSequencerProfile<=0)
    return "*";

  AUTO_LOCK(m_csMapOpenFiles);

  static string szFileName;

  std::map<FILE*,SFileAccessStats>::iterator iter = m_mapOpenFiles.find(hFile);
  if(iter == m_mapOpenFiles.end())
    szFileName = "";
  else
    szFileName = iter->second.sName;

  return szFileName.c_str();
}

CFileReadSequencer::SFileAccessStats * CFileReadSequencer::GetFileInfo(FILE * hFile)
{
  if(m_nSequencerProfile<=0)
    return NULL;

  AUTO_LOCK(m_csMapOpenFiles);

  std::map<FILE*,SFileAccessStats>::iterator iter = m_mapOpenFiles.find(hFile);
  if(iter == m_mapOpenFiles.end())
    CFileReadSequencer::Log( "Reading from undefined file" );

  return &m_mapOpenFiles[hFile];
}

void CFileReadSequencer::LogStatus(const char * pCommandName, FILE * hFile)
{
  const char * pFileName = GetFileName(hFile);

  CFileReadSequencer::Log( "%s, %s, %s, %.1f (%.1f/%.1f/%.1f) MBytes, %d (%d/%d/%d) reads, %.1f (%.1f/%.1f/%.1f) sec, %s", 
    pCommandName, m_sSectionName.c_str(), (m_nSequencerMode == 2) ? "CREATE" : "READ", 
    float(m_seqStats.ok.nBytes)/1024/1024, float(m_seqStats.skip.nBytes)/1024/1024, float(m_seqStats.oth.nBytes)/1024/1024, float(m_seqStats.off.nBytes)/1024/1024,
    m_seqStats.ok.nReads, m_seqStats.skip.nReads, m_seqStats.oth.nReads, m_seqStats.off.nReads, 
    m_seqStats.ok.fTime, m_seqStats.skip.fTime, m_seqStats.oth.fTime, m_seqStats.off.fTime, 
    pFileName );
}

CFileReadSequencer::CFileReadSequencer()
{
	m_sSectionName.reserve(128);
}

void CFileReadSequencer::Init()
{
  AUTO_LOCK(m_csFread);

  m_nSequencerMode = gEnv->pConsole->GetCVar("sys_FileReadSequencer")->GetIVal();
  m_nSequencerProfile = gEnv->pConsole->GetCVar("sys_FileReadSequencerProfile")->GetIVal();

  if(m_nSequencerProfile==0)
    m_mapOpenFiles.clear();

  if(m_nSequencerMode || m_nSequencerProfile)
    CryLogAlways( "FILE_SEQUENCER: MODE = %s%s", (m_nSequencerMode == 2) ? "CREATE" : "READ", m_nSequencerProfile ? ", PROFILE" : "" );
}

void CFileReadSequencer::StartSection( const char * pSectionName )
{
  if(gEnv->IsEditor())
    return;

  AUTO_LOCK(m_csFread);

  m_sectionStartTime = gEnv->pTimer->GetAsyncTime();

  // reset stats
  m_seqStats.Reset();

  assert(pSectionName);

  if(strstr(pSectionName,"/"))
    m_sSectionName = strstr(pSectionName,"/")+1;
  else
    m_sSectionName = pSectionName;

  if(!m_nSequencerMode)
    if(!m_nSequencerProfile)
      return;

  m_dwTargetThread = GetCurrentThreadId();

  CloseSeqFile();

  if(m_nSequencerMode==1) // load from sequencer file
  {
    SAFE_DELETE(m_pReadThread);
    char szFileName[512];
    sprintf_s(szFileName, "%s\\%s.seq",
      gEnv->pConsole->GetCVar("sys_FileReadSequencerFolderNameRead")->GetString(), CFileReadSequencer::m_sSectionName.c_str());
    m_pReadThread = new CReadSequencerThread(szFileName);
  }

  if(m_nSequencerMode==2)
  {



    if(m_hWriteFile)
      ::fclose(m_hWriteFile);
    char szFileName[512];
    sprintf_s(szFileName, "%s\\%s.seq",
      gEnv->pConsole->GetCVar("sys_FileReadSequencerFolderNameWrite")->GetString(), CFileReadSequencer::m_sSectionName.c_str());
    m_hWriteFile = ::fopen(szFileName,"wb");      
    m_nCurPageStartOffset = 0;
    m_nCurPageBytes = 0;
  }

  LogStatus("SECTION_START", NULL);
}

void CFileReadSequencer::UpdateCurrentThread( bool bUpdateFromNextFileRead )
{
  AUTO_LOCK(m_csFread);

  if(!m_nSequencerMode)
    if(!m_nSequencerProfile)
      return;

  if(bUpdateFromNextFileRead)
    m_dwTargetThread = ~0;
  else
    m_dwTargetThread = GetCurrentThreadId();
}

float CFileReadSequencer::GetSectionTime()
{
  return (gEnv->pTimer->GetAsyncTime()-m_sectionStartTime).GetSeconds();
}

void CFileReadSequencer::EndSection( )
{
  if(gEnv->IsEditor())
    return;

  AUTO_LOCK(m_csFread);

  CloseSeqFile();

  /*if(m_nSequencerMode==2 && strcmp(m_sSectionName.c_str(), "SYSTEM"))
  {
#ifdef XENON
    DmMapDevkitDrive();
#endif

    char szFileNameDone[256]="";
    sprintf_s(szFileNameDone, "%s\\%s.seq",
      gEnv->pConsole->GetCVar("sys_FileReadSequencerFolderNameWrite")->GetString(), "DONE");

    if(FILE * fD = ::fopen(szFileNameDone,"wb"))
      ::fclose(fD);
  }*/

  if(m_nSequencerMode || m_nSequencerProfile)
    LogStatus("SECTION_END", NULL);
  else
    CryLogAlways("======== %s is loaded in %.1f sec ========", m_sSectionName.c_str(), GetSectionTime());

  for(std::map<FILE*,SFileAccessStats>::iterator iter = m_mapOpenFiles.begin(); iter != m_mapOpenFiles.end(); iter++)
  {
    SFileAccessStats * pInfo = &iter->second;

    if(pInfo->ok.nReads || pInfo->oth.nReads || pInfo->off.nReads || pInfo->skip.nReads)
      CryLogAlways("%.1f / %.1f / %.1f / %.1f sec, %.1f / %.1f / %.1f / %.1f MB, %d / %d / %d / %d reads, %s", 
        pInfo->ok.fTime, pInfo->skip.fTime, pInfo->oth.fTime, pInfo->off.fTime, 
        float(pInfo->ok.nBytes)/1024.f/1024.f, float(pInfo->skip.nBytes)/1024.f/1024.f, float(pInfo->oth.nBytes)/1024.f/1024.f, float(pInfo->off.nBytes)/1024.f/1024.f, 
        pInfo->ok.nReads, pInfo->skip.nReads, pInfo->oth.nReads, pInfo->off.nReads, 
        pInfo->sName.c_str());
    
    ZeroStruct(pInfo->ok);
    ZeroStruct(pInfo->oth);
    ZeroStruct(pInfo->off);
  }

  for(int i=0; i<m_arrDeferredBlocks.Count(); i++)
    SAFE_DELETE(m_arrDeferredBlocks[i]);
  m_arrDeferredBlocks.Reset();
}

IFileReadSequencer * CFileReadSequencer::GetFileReadSequencer()
{
  static CFileReadSequencer s;
  return &s;
}

void CFileReadSequencer::Log(const char * szFormat, ...)
{
  va_list	arglist;
  char szMessage[512];
  va_start(arglist, szFormat);
  vsnprintf_s(szMessage, sizeof(szMessage), sizeof(szMessage), szFormat, arglist);
  va_end(arglist);	

  CryLogAlways( "FILE_SEQUENCER(%2d, %.1f s, %d MB, %d def): %s",            
    CFileReadSequencer::m_nMessageOrder++, CFileReadSequencer::GetSectionTime(), 
    int(m_seqStats.ok.nBytes+m_seqStats.skip.nBytes+m_seqStats.oth.nBytes+m_seqStats.off.nBytes)/1024/1024, m_arrDeferredBlocks.Count(),
    szMessage );
}
